# -*- coding: utf-8 -*-
import time, json
import scrapy, re


class SohuSpider(scrapy.Spider):
    name = 'qie_channel'

    def start_requests(self):
        urls = {
            '3': "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=1", #娱乐
            '9':'https://pacaio.match.qq.com/xw/site?uid=1_563585538&ext=ent&num=20', #影视
            '78': 'https://pacaio.match.qq.com/xw/site?uid=1_563585538&ext=tech&num=20',  # 科技
            '86': 'https://pacaio.match.qq.com/irs/rcd?cid=56&ext=society&token=c786875b8e04da17b24ea5e332745e0f&num=20',  # 生活
            '2': 'https://pacaio.match.qq.com/irs/rcd?cid=56&ext=baby&token=c786875b8e04da17b24ea5e332745e0f&num=20',  # 育儿
            '13': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=society&page=1',  # 社会
            '11': 'https://pacaio.match.qq.com/irs/rcd?cid=56&ext=history&token=c786875b8e04da17b24ea5e332745e0f&num=20',  # 历史
            '84': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=emotion&page=1',  # 情感
            '5': 'https://pacaio.match.qq.com/xw/site?uid=1_563585538&ext=auto&num=20',  # 汽车
            '1': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=games&page=1',  # 游戏
            '7': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=food&page=1',  # 美食
            '88': 'https://pacaio.match.qq.com/irs/rcd?cid=92&token=54424c1ebe77ea829a41040a3620d0e7&ext=fashion&page=0',  # 时尚
            '14': 'https://pacaio.match.qq.com/xw/site?uid=1_563585538&ext=sports&num=20&expIds=&page=0',  # 体育
            '12': 'https://pacaio.match.qq.com/irs/rcd?cid=56&ext=health&token=c786875b8e04da17b24ea5e332745e0f&num=20',  # 健康
            '4': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=visit&page=1',  # 旅游
            '61': 'https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=finance&page=1',  # 财经
            '90': 'https://pacaio.match.qq.com/irs/rcd?cid=135&token=6e92c215fb08afa901ac31eca115a34f&ext=digi&page=0&num=15',  # 数码
            '8': 'https://pacaio.match.qq.com/irs/rcd?cid=56&ext=house&token=c786875b8e04da17b24ea5e332745e0f&num=20',  # 房产

        }
        for cate in urls:
            item = {}
            item['site_classify_id'] = cate
            yield scrapy.Request(urls[cate], meta={'item': item}, )

    def parse(self, response):
        item = response.meta['item']
        cate = item['site_classify_id']
        data_list = json.loads(response.text)
        for data in data_list['data']:
            item = dict()
            item['publish_time'] = data.get('update_time')
            item['url'] = data.get('url')
            item['title'] = data.get('title')
            item['spider_time'] = time.strftime("%Y-%m-%d %X", time.localtime())
            item['site_id'] = "2"  # 站点分类
            item['site_classify_id'] = cate
            item['author'] = data.get('source')
            item['browse_count'] = data.get('view_count')
            item['comment_count'] = data.get('comment_num')
            video_count=data.get('ext').get('video_count')
            if video_count!=0:
                continue
            if 'video' in item['url']:
                continue
            # print(item)
            yield item
